{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Importing TensorFlow and IMDb dataset from keras library\n",
    "from keras.datasets import imdb\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.3.0\n"
     ]
    }
   ],
   "source": [
    "# Checking TensorFlow version\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.ops import rnn, rnn_cell\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating Train and Test datasets from labeled movie reviews \n",
    "(X_train, y_train), (X_test, y_test) = imdb.load_data(path=\"imdb_full.pkl\",\n",
    "                                                      num_words=None,\n",
    "                                                      skip_top=0,\n",
    "                                                      maxlen=None,\n",
    "                                                      seed=113,\n",
    "                                                      start_char=1,\n",
    "                                                      oov_char=2,\n",
    "                                                      index_from=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = [item for sublist in X_train for item in sublist]\n",
    "vocabulary = len(set(t))+1  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fd27561b908>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl8VPW9//HXR3Bp3ZVFBBW1WMRaUanicutC3bCt+qv1avtTamvx3qu3tnp/V1yqVqVqFeuGWhesWJeiYkVAFiGo7IR9hwQCCWsCIQkJ2b+/P+YkTpKZzEwyW+a8n49HHpn5zjlnvt+ZM+dzvsv5HnPOISIi/rNfqjMgIiKpoQAgIuJTCgAiIj6lACAi4lMKACIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj7VOdUZaE2XLl1c7969U50NEZEOZeHChUXOua6RlkvrANC7d2+ys7NTnQ0RkQ7FzDZFs5yagEREfEoBQETEpxQARER8SgFARMSnIgYAMzvOzLLMbJWZrTSzu7z0R8xsi5kt8f4GB61zn5nlmNlaM7siKP1KLy3HzIYlpkgiIhKNaEYB1QL3OOcWmdmhwEIzm+q99lfn3DPBC5tZP+BG4DTgWOALMzvFe3kkcBlQACwws3HOuVXxKIiIiMQmYgBwzm0DtnmPy8xsNdCzlVWuAT5wzlUBG80sBzjHey3HObcBwMw+8JZVABARSYGY+gDMrDdwJjDPS7rTzJaZ2SgzO9JL6wnkB61W4KWFSxeRDmrFlhKW5u9JdTakjaIOAGZ2CPAx8HvnXCnwCnAy0J9ADWFEPDJkZkPNLNvMsgsLC+OxSRFJkB+/OJNrRs5KdTakjaIKAGa2P4GD/7vOubEAzrkdzrk651w98DrfNPNsAY4LWr2XlxYuvQnn3GvOuQHOuQFdu0a8kllERNoomlFABrwJrHbOPRuU3iNoseuAFd7jccCNZnagmZ0I9AHmAwuAPmZ2opkdQKCjeFx8iiEiIrGKZhTQBcDNwHIzW+Kl3Q/cZGb9AQfkAbcDOOdWmtkYAp27tcAdzrk6ADO7E5gMdAJGOedWxrEsIiISg2hGAc0ELMRLE1tZZzgwPET6xNbWExGR5NGVwCIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj6lACAi4lMKACIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj6lACAi4lMKACIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj6lACAi4lMKACIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj6lACAi4lMKACIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj6lACAi4lMKACIiPhUxAJjZcWaWZWarzGylmd3lpR9lZlPNbL33/0gv3czsBTPLMbNlZnZW0LaGeMuvN7MhiSuWiIhEEk0NoBa4xznXDxgI3GFm/YBhwDTnXB9gmvcc4Cqgj/c3FHgFAgEDeBg4FzgHeLghaIiISPJFDADOuW3OuUXe4zJgNdATuAZ421vsbeBa7/E1wGgXMBc4wsx6AFcAU51zu51zxcBU4Mq4lkZERKIWUx+AmfUGzgTmAd2dc9u8l7YD3b3HPYH8oNUKvLRw6SIikgJRBwAzOwT4GPi9c640+DXnnANcPDJkZkPNLNvMsgsLC+OxSRERCSGqAGBm+xM4+L/rnBvrJe/wmnbw/u/00rcAxwWt3stLC5fehHPuNefcAOfcgK5du8ZSFhERiUE0o4AMeBNY7Zx7NuilcUDDSJ4hwKdB6bd4o4EGAiVeU9Fk4HIzO9Lr/L3cSxMRkRToHMUyFwA3A8vNbImXdj/wJDDGzH4DbAJu8F6bCAwGcoAK4FYA59xuM3sMWOAt96hzbndcSiEiIjGLGACcczMBC/PyoBDLO+COMNsaBYyKJYMiIpIYuhJYRMSnFABERHxKAUBExKcUAEREfEoBQETEpxQA0lRhWRU7SitTnQ2Jk9m5RSzeXJzqbIg0Ec11AJICPxj+BQB5T16d4pxIPPzi9XmAvk9JL6oBiIj4lAKAiIhPKQCIiPiUAoCIiE8pAIiI+JQCgIiITykAiIj4lAKAiIhPKQBI2hmZlcO1I2elOhsiGU9XAkvaeXry2lRnQcQXVAMQEfEpBQAREZ9SABAR8SkFABERn1IAEBHxKQUAERGfUgCIo+raep6ZvJbyqtpUZ0VEJCIFgDgak53PS1k5vDBtfaqzIiISkQJAHFXX1gNQ5f0XEUlnCgAiIj6lACAi4lMKACIiKXDLqPlc+syMlOZBk8GJiKTAV+sKU50F1QAkM7w4bT29h03AOZfqrIh0GBEDgJmNMrOdZrYiKO0RM9tiZku8v8FBr91nZjlmttbMrghKv9JLyzGzYfEvivjZiKnrUp2FlHHO8afPVrI0f0+qsyIdTDQ1gL8DV4ZI/6tzrr/3NxHAzPoBNwKneeu8bGadzKwTMBK4CugH3OQtKyLtVFVbz1uz8rjhb3NSnRXpYCL2ATjnvjKz3lFu7xrgA+dcFbDRzHKAc7zXcpxzGwDM7ANv2VUx5ziNmaU6ByIi0WtPH8CdZrbMayI60kvrCeQHLVPgpYVLb8HMhppZtpllFxamvpNERCRTtTUAvAKcDPQHtgEj4pUh59xrzrkBzrkBXbt2jddmk0L9jyLSkbRpGKhzbkfDYzN7HRjvPd0CHBe0aC8vjVbSRUQkBdpUAzCzHkFPrwMaRgiNA240swPN7ESgDzAfWAD0MbMTzewAAh3F49qebRFpThVQiVXEGoCZvQ9cDHQxswLgYeBiM+tPYJ/LA24HcM6tNLMxBDp3a4E7nHN13nbuBCYDnYBRzrmVcS+NiIhELZpRQDeFSH6zleWHA8NDpE8EJsaUOxERSRhdCSwiaaG+3lFTp6nUk0kBII50HYD4VTzugveHMUvo88DncciNREsBQOLm5jfn8ehnGXVtn0TpuS/aPxXHp0u2xiEnEgsFAImbr9cXMWrWxlRnw79SOAyopk5jkDoiBQCRDk5Nj9JWCgAiIj6lACAiLczKKaJWI3IyngJAHGkuIMkEczfs4pdvzOP5aetTnRVJMAUAEWmisKwKgA1F5SnOiSSaAkAc+bkzrqSiJtVZ6LBGZuXw4xe/bvd2nGYDkhgpAEhcDH0nO9VZ6LCenryWFVtKW6TvLq/mvrHLqKypa3V9I/VnHn4++enIFAAkLnIL96Y6Cxnn6clreH9+Pp8s1szpkhgKACJpSoMKJNEUAEREfEoBQETabXd5daqzIG2gACC+sL2kssNd2BRrE1Aqm4w0kVvHpACQAE6Nt2llT0U1A5+YxmPjO+ZMpZEG2CRqBM6EZdso2luVmI1LWlAAiCONhEtPJfsC1yhkrS1McU46nue/0NXAmUwBQETEpxQA4sjfDT+q/8SbruyVRFMASADTZZEptWlXOYs3F6c6G3GTiN1pZ1klf/psZYfrGJf4UgCQjHPR0zO47uXZqc5G0sVSX7h/7ArempXH1+uLEpYfSX8KACI+VFcfOPMP1cykhif/UAAQSVPRjibWqGNpKwWABIj2OoDZuUXk765IcG5CW1awh10a490hRDvbp3qeJFYKAHEU6w/wF6/P46KnsxKSl0h++tIsfvzizJS8t6SPUOcqwfuxxjMkV3F5NTvLKpP2fgoAKVafwur7tpL47WiJPlB8vLCAnyhgxY1GqkVn867k1tDPfGwq5wyflrT3UwCQDuGeD5eyfEtJqrORVLGeG2RaV8CZj07h4hTVkAFm5RTxw6ez+GRxQcrykGgKAJJRMrJD1Kcn68UVNeQl+Qw82NrtZQAszW954lEfouq+aVc5Yxd1rGChACDiYxkZMBNs7fYyTrp/IlNX7WiS/uMXZnL3mKUpylXbZHQAyM7bzYy1OxP6HhuLylucDUxP8Ht2BImaH35FB2oGWpq/h20l++K6zW0l+1i8uZjXv9oQ1+2mu6raOvZW1SZs+2u3l/G9hyezPYp+sSX5gavMp67a3iS9LIH5S5SIAcDMRpnZTjNbEZR2lJlNNbP13v8jvXQzsxfMLMfMlpnZWUHrDPGWX29mQxJTnKauf3UOv3prQVTL7iitjLn3ffW2Ui55ZgavfJkLfNMGm797Hws3FbNlT+gf/xl/msKD/1oe03tF4pyjPE12wBVbSjjrsal8mJ0fdpnV20rZUdr65z07J3CVavBN0SONXNrjzfwZSvBFT5U1dczJ3dXqtqK1vaSS3sMmsHBT0+knrhk5iwufan8bdum+Gr7wzjbPe2I61708m+ETV7d5e5t3VTB9TeAkxSww8qSguCKh+88rM3L5x9xNbV7/2pGz+d7Dk6mpq2+yPzT454LN7QqKo+fksbeqlqmrA5/z7NwiHu2g04fHIpoawN+BK5ulDQOmOef6ANO85wBXAX28v6HAKxAIGMDDwLnAOcDDDUEjWbaXVDJvQ+AHX1ZZ0yLSn/vnaY297wXFFVRUt/wxlFbWcPULXzfeAP3z5dsAWLSp5bwz78/fzAVPTuerdS2nIC7ZV8M/5m5uUzn2VtXy2PhVLX4E78/P57SHJ5NXVA4EDnBL8vdEtc2Z64u44W9zmL9xd8Rlq2vreXz8Kkoqwh9oP18R+Fz+30fL+OvUdWStaVkjuur5rxn4RMvRDlW135Sr4XPu+8dJEfPVINSw2lDj6B8dv4qbXp/b2M7boLKmjrLKGjYWlVMTZp6cqto67v7nksaz+9m5gUD1j7mbGtdvUNesdjhoxAx+9/7ikNvNLdzLvR8ta1ynoXnm8QmruW10dsQTlObvFc7t/1jY5PmZj03lwqeyOO3hyRE7PKtr6+k9bALvzMlrdbm8ovImv7GnJq3hwX+t4LqXZ7E0yv2ywQOfLGf1tlIABj//dYv9IX93Bfd+vJzhE1ezbkcZE5Zta9E8E6tHxq0Mme6cY8rKltsO/uzLKmuorKmjrt41pi/J3xP29+icY8io+Tz06YqQrydSxADgnPsKaH5kuAZ423v8NnBtUPpoFzAXOMLMegBXAFOdc7udc8XAVFoGlYQa+MQ0/v21uRSWVXHZs181Ofhs9A6aDS58Kotb3pzfYhujZ+excmspg0Z8CcAL03MAmLZmJ2u2l7IpqMOq4cset3Qrny2N392SXs7K4c2ZG1ucTTVURzcUBQ6awz5exrUjZzU5y66rdwwdnd1iorQ73lvE/I27ueFvcyIGjfHLtvLGzI08OSn8GejIrNzGx89PW8+tfw/UwuZv3M2nS7Y0vhaq/Tk4WBQUR9d8EnzTkrLK8Gex+bv3NZ71r98ROPCX7KvhuS/W0XvYBHaWVnLhU1mc/sgULnlmBn0e+LzJwTw4j2MXb2k8SAS3+w4a8SWnPzKlSe0vOFjnFpYzztsfsvN281/vfnMwvvO9xfwzO58120tD5r+6tmVAWl5Qwm2js8OWOdhL09fzg+FfNGmybP4dzIhwz4SGZphnp65rdbmLn5kRMsAv3ryHRz4LfG5TVm5vEYBDeXfeNydL63fubfH6vqDPt6qmnjveW8RvW/lMXpmRS+9hE/jnguhOwpxzbN5VwZzcXWSt3ck0bx8N/uxOvn9i4+PTH5lC3z9O4uT7J3LWY1MBuHbkLK4dOQuADYV76T1sQuPy9Q6+XFfI6DltryG1VVv7ALo757Z5j7cD3b3HPYHgen+BlxYuvQUzG2pm2WaWXVgY/xt4TFqxje3Nmh6u+OtXjY8bzk6yNxUzO7eIrKD2/Pl54WeY/Pmrc/j77LzG5znejvrRwgL+O8wZXzgNZ75AiwNQrffjjXS2t7Qg0FYeXK3fumcfU1btaJKf3MK9jTdMASgsCxxM35q1sUWTRvD719Q1ff9oBqrc8Lc53PXBkhbpHy8soDhEn8HfvtrAU5PWtEifk7urydXWvw+xTaDJj6zBTa/PBZr+eJ/zbnpyzp+ntbgD1ucrmrbzBiuuqGlyUHbONR74L3hyemP6njC1pdtGZzNxefjtR+PuMUtC1jKbq66t55kp6xq/3/YqDlOmZQV7+E7QwTD0MoF9c+g7C7niua9aXTacUMEQYGwUQzYb9qmnJq1tTGvYHWpq60OO8Pnh01nc9PpcivbG1rdV0qxJclvJPhbkRa5pF+2tirpG1x7t7gR2gV9i3HLqnHvNOTfAOTega9eu8dpsoz9+2rRqN3H5NqqDqvrXeFEaAlfq3hrUh1DQyrQN8RxN0VDDALhvbOS+gtzCvS1qMdH61+ItIdP/9NkqfvbKbH7+6mxGz8kLeSYcDxuLyrnnw6X87oPQQfLNrze2SLvp9bn8K6gmMTOn7TNaTg/RPBWt+Rt3c8qDn7d5/UTYWVaJc46xiwqaNKcF5zPSNWAfBw1lHLtoS5PpSiJNc/LWrLzGk4Rw4nFgGzF1bcj0UCcSkRSXV1PqHagfHb+KB/61Iuzv+dUvc0O/0IrgWu95T0xncxTTvwx4/Auenhy6jPHU1gCww2vawfvf8CvaAhwXtFwvLy1cesr917uLol52QxsPsg1q6+pDnrnMauUAFs09WQeN+DLseOmSfTUsL4ht5MzCTd+coSzIK+ahT1dy+iNTmvSLVNXWx2WkT8NBamdpoJxTomy7zd8dXfPQvuq6Vme8bMsPOlki3RDm9neymzR/ANTWOaat3sndY5YyYkrrzTThBDcD7a2qTcjU2qFqZw1m5RRFrKls3fNNLb69J19nPjaV8cu2NT5/f37TpqHgzW8oLA+Z3prmtd7gJtLWTFvdvn6MaLQ1AIwDGkbyDAE+DUq/xRsNNBAo8ZqKJgOXm9mRXufv5V5ah7ChsGW7Y3ORzozKKmu47uXZIc8Yf/nGvMbHN702N/YMtpKf616ezU9eim0KhZ+9MidkenlVXeNe/9nSrZz12FRq6upbVHNDmR0myDX/2CY1a3KpbucNS059aBJvzcpr8/r/+9Ey3pu3md7DJjS2f4f7qsPtAdEOBW3o6PwyiiYdgMkrd4TsJyn1amvRNPeMaWWkVoPgk5BkXDbwyzfm8fNXIwedrXv2tRgMEW7kXSitVYSiLadzLmSTUSyinTwyETpHWsDM3gcuBrqYWQGB0TxPAmPM7DfAJuAGb/GJwGAgB6gAbgVwzu02s8eAhvaUR51zkRvC4iTUiB4g5HCyUK547ivWDx/crjwM+3h5VFMZzNkQn6GJ0WrvvtfngeiaQH4RFORCja5J5NQ0wWd3DWL50d3/SaAZrqC4gr7HHMZ/xlBrBGI+gw53rUOs31WkphhoWeNKxUXHzjle/7rpEM5orgA+/8npXNq3G8Ou6tuYFsv8Vq3tc8H7R2uf+81vzm9XE2SqRQwAzrmbwrw0KMSyDrgjzHZGAaNiyl2c9HsodGUjeHRBa5p3eLZF847naDUMYVxeUNKk8yhcjn7992xeuOnMkK9Fc0BIhuBhdG0NQKmaHaEj3EKxYb/+bOlWXgyzLyRK8+8lf3cFd7zXesD8cGEBf57YsrM/GtPX7GwSACLZGlRDKNpbzaQVLU8OouVc+/qfIm4/YVv+RsQAkMliOQsMN+qgcVsR1g81oiYaDe3ADc04t//wpIjr3D92OUcfckCL9Ge8TqXgZoOXsnKaLLO9leYKh4vLkbe+lc89EbXhUGd6bXmb8qpavtNKjSdeeW/cThu2t3Z7WZv3tWjEWsaXZ+Q2jvoJJ9rhvvFwftDoLID/+Efk2ly4WkWkCxk7Al8HgNYORM09HObCkGQIbsttzzEmmual5qOkWkiPSkRKhOsbibe8XRX0HjaB44/6dszrVlRHbtZM1FTQrV0cmEjBv+OtMfQBtCZ4N/8iTGdsvM7+U/mTyti5gBqu0m1NLFcLhrraNxkW5BXzg+FftEhvtQMrzWf4Cj7+NB/pEu2xKZZjWKhF0/kjaugMjma4YDKYfXOFdKh9scGZj01pUw0xN8TFXbF4cdo3tdg0aeWMi2T8jjO2BhBNR11lTfzacxP1XcXzYpB0nlW4rWel0XbkJ0Os31RtHPqW4i3c99D3j5PYv1Pr31HoXTVyGSdEcbKWyPVDSr+vJiEyNgAkWlaKZ/x84vM1XNq3G5f9NfSVlAkJSHHaZvDcPA35dM7x29HZUTVhADwzZR17KmoaJ+9q9f2SFPlinVwukbNbxlPDd9SWwRDvz488zDQd+eT47+8AEFMzQrNlb202y2iki3YieePrDfQ6MnKbb/BEWn9uZUbI9uYn2coqa2OewOuNmS2vEk6laC7aS7RFmyM3VTY0MfmFc45ZObu44DtHpzoracfXASDS6IRkenxCdNP7zguasTMrzMRdHevQnxod8TP6t79Enlr6zRQHxbGL0uIC/yY+WbyFu8cs5S8/+36qsxJSrBcWxlPGdgInWzz7E+IhnTs5Q9W8EtlME2o6aPGPhmlDYhm1U1ufXr/nRFEAyFDNO4+nrNze7jOKeMWU4FEfuVFMs9Febb0ILxMUFFewL8p+FUjvgQJt1dAcOi6GadmjnWuqo/N1E1Cmqqiu47CD9m+SNvSdhRx1cMuLw2Lx8aICOsXhVH1E0FzyDRNlJaujtlE6V5Hi6MKnsjj/5MS0fXeUTuxPwsx4my6CZ21tIgm7qAKAj7R39s6/TEr89LQSf7NjGJ00NoaD5V0x3ucinmIZI78pinmFUmnIqJY3n0oWNQFlqI42CijZ7fSp/nSaTzncEU1rx70U4iXpNccEWLQ5tltkxpMCgKSFZP6QP1pYkPIRYNHc6Efap6q2Lil31erI1AQkaSGZ1fT/+XBp5IWkw/vug5MY1LdbqrPRZhoGKiISQuOEqRGOkunQTJXOFACitGZ7WaqzEJMdpam/KlVE0psCgIh0OBNC3OUt0yRjNlAFABERn1IAEBFJQ4m6cU8wBQAREZ9SABARSUNVSbjZkQKAiEga2hrmZvTxpAAgIuJTCgAiIj6lACAi4lMKACIiPqUAICLiUwoAIiI+pQAgIuJTCgAiIj7VrgBgZnlmttzMlphZtpd2lJlNNbP13v8jvXQzsxfMLMfMlpnZWfEogIiItE08agCXOOf6O+cGeM+HAdOcc32Aad5zgKuAPt7fUOCVOLx3SMmYRlVEpKNLRBPQNcDb3uO3gWuD0ke7gLnAEWbWIwHvLyIiUWhvAHDAFDNbaGZDvbTuzrmGuzVsB7p7j3sC+UHrFnhpcacKgIhIZO29KfyFzrktZtYNmGpma4JfdM45M4vpcOwFkqEAxx9/fDuzJyIi4bSrBuCc2+L93wl8ApwD7Gho2vH+N9yVeQtwXNDqvby05tt8zTk3wDk3oGvXrm3LV5vWEhHxlzYHADM72MwObXgMXA6sAMYBQ7zFhgCfeo/HAbd4o4EGAiVBTUVxpU5gEZHI2tME1B34xLttWWfgPefcJDNbAIwxs98Am4AbvOUnAoOBHKACuLUd790qHf5FRCJrcwBwzm0AzgiRvgsYFCLdAXe09f1ERCS+MvJKYLUAiYhElpEBQEREIsvIAODUCyAiElFmBgAd/0VEIsrIACAiIpEpAIiI+FRGBgA1AYmIRJaRAUBERCLLyACgUUAiIpFlZAAQEZHIMjIAqA9ARCSyzAwAqc6AiEgHkJEBQEREIsvIAKD7AYiIRJaRAUBERCLLyACg838RkcgyMgBU19anOgsiImkvIwNA6b6aVGdBRCTtZWQAEBGRyDIyAHg3qhcRkVZkZAAQEZHIMjIA6PxfRCSyjAwAIiISWUYGgIMP7JzqLIiIpL2MDABHfHv/VGdBRCTtZWQAEBGRyBQARER8SgFARMSnFABERHxKAUBExKcyMgDU64YwIiIRJT0AmNmVZrbWzHLMbFgi3qNTnOcC+vA/zovLdm465zhuu/DEFun/PuA4/s9ZPVukH3xAp8bHf/jRKUy/56KQ2x35i7OizsMffnQKA086Kurl4+GuQX2S+n7h/PSMY2Ne5y8/+36T5z0OP6jN7z/nvkvbvG4oPQ4/iNOOPaxd23jw6lMBMINfnnt8q8ve+IPjwr72q/N7c0Dn2A4n5598NGNuD/3b+sOPTgm73nP/3p8Phg6M6b2C/ejUblEv++sLWv5eux16YIu0vsccyvz7B4XcxuDTj+H/XfHd6DMIDDzpKNY9flVM67RFUq+YMrNOwEjgMqAAWGBm45xzq+L5Pp077Ufunwczfc1Ofjs6m0v7duMv13+fAY9/wZHf3p8v//cSvv/IFADO6HU4/7z9PF6ansNLWTkA/O3ms7n9nYVc2/9YHvrJaRx18AHk/nkwBhSVV3Fgp07U1Ndz+zsLWbipmN8N6sP1Z/Xih09n8W99uvDOb84FoK7eUVZZw6UjvuTBq0/l6u/34MDOnbjq9GP42StzeOtXP+CSvt/sjIVlVXy9voj1w69i/06BH1NlTR27yqvpecS3ABj/3xeSv7uC/3x3UeN6g08/hgevPpWf9j+WunrHnooaTu0RODAMeHwqRXur+cdvzuXCPl0AuIs+7Cit5JdvzOOhH/fjzvcWUVpZC0Dek1czdlEB3Q49iP/75jwgcJAYcn5v+jzwOQCv/PKsJu///I39ueuDJY3PB/Xtxs3nncDAk47moP0DQeyW805gV3k1+3faj9J9NczMKeK3/3YSB3Tej7p6x8n3T2x8/8qaOi56Oovh155O/+OPYEdpJc7B93oezthFBXQ/7CC+Xl9E76O/zRerd/LF6h0AHP6t/Xn0mtO4+vQerN1RxpzcXTw+YXVjvu689DuMW7qVy/t1Z8qqHdx92Slc2rcb20squW10NvcP7suJXQ5hUN9unOTl54YfHMf9nyzn7BOO5L7Bp9L/uCMAKKmo4YxHpzDtnosYNOJLALIf/BHzNuzmjvcW0f2wA3ng6n6U7qvhsn7dqaypo8fh3+KPP+7HY+NXcVKXg9lQVM73ex3OCzeeycXPzABg+j0XsWl3BX2POZRFm/awdc8+DjqgEz8/uxc5O/fyype5/OKc47ngO10ay3Xlc1+xZnsZAI9ecxq3nNebrLU7ufWtBVx0Slde/MWZ7GfGh9n5/OmzVUy75yK6H3YQnfczDuy8H4VlVQw+vQdnHHcEw687nZJ9NYxbsoWLTunGT0fO5Jnrz+BH/boD8MGC/Mb3Hf3rc7hl1HwAHvnpaTz8k358uLCA//1oWYvf5HVn9mTEz89gQd5+LhIiAAAIA0lEQVRuXvtqA1ed3oPrz+5FZU1d4zIbnxjMw+NWMnrOJq4f0It1O8r4af9jWb2tlB6HH8S9Hy8H4NozAydLHwwdyKEHdWba6p08O3UdE353IacdezjOOerqHZ077cf6HWXUO1i1rYTzT+7CYQftz7cO6ETvYROa5K/nEd/iZ2f34tK+3aiuraf30d+m22EHNe7/APvtZ0xZuZ3L+nVnd3k1Fz89g/uvPpVLvtuNY8KcGOQMv4rO3m/5yu8dQ1llLUs2F5O3q4K/z84DoMshB/D8jWfy6pe51NY55mzYxbX9e8YcUNvEOZe0P+A8YHLQ8/uA+8Itf/bZZ7v22F6yz51w73g3ecU255xz45dudZt3lTvnnNtTXu3uen+RKyyrbFx+5vpCV1tXH/X2n5m8xp1w73i3sXCvc865eRt2uX3VtVGtW15VE/X7hFJfX+++99Ak9+yUta0ut6+61o1fujXi9lZtLXErt5Q0SXti4mp3wr3j3VszNzjnAp9Z/u7A5/fQv5a7E+4d7+4fu8zV19e7cUu2uIc/XeHm5ha1qTz7qmvdrr1VbVp38eZil7uzLORr94xZ4vr98fPG51v3VLj6+vrG/+FsKip3pfuqo3r/E+4d7064d7xzLvAZnXDvePcf72S3us6abaXuhHvHu6GjFzjnnPvv9xY1bqOtgstUuq/aXfpMlltesKdd22zuV6PmNeZz3fZAGW57e0GTZSat2OY+ys5378/b5E64d7y7+OmsVre5r7q28XdXXVvn1m4vDblcVU2dq66ta5FeX18f8+9p865yt2RzsVuaX+xemr6+1X0hFos3F7vlBXvcoBEz3OLNxa0uW1tX7/KK9jY55lTV1Lm3Zm6I6TgUCpDtojgmm0tie7mZXQ9c6Zy7zXt+M3Cuc+7OUMsPGDDAZWdnJy1/saqrdxQUV3DC0QenOisJUV5VywvT1nP35adwYOdOTV6rqavnmSlrueOS73DYQf6+8nruhl3U1zvO987KV24t4aQuh/CtAzq1ut6nS7ZwSd9uHHbQ/t4PMnCWmc4qa+rYXV7NsV6NdNzSrVz83a5h94GdpZV8+8DOHKLpWZLKzBY65wZEXC7dAoCZDQWGAhx//PFnb9q0KWn5ExHJBNEGgGR3Am8BgnuSenlpjZxzrznnBjjnBnTt2jWpmRMR8ZNkB4AFQB8zO9HMDgBuBMYlOQ8iIkKSRwE552rN7E5gMtAJGOWcW5nMPIiISEDSe2accxOBicl+XxERaSojrwQWEZHIFABERHxKAUBExKcUAEREfCqpF4LFyswKgfZcCdYFKIpTdjoKv5XZb+UFldkv2lPmE5xzES+kSusA0F5mlh3N1XCZxG9l9lt5QWX2i2SUWU1AIiI+pQAgIuJTmR4AXkt1BlLAb2X2W3lBZfaLhJc5o/sAREQkvEyvAYiISBgZGQCScd/hZDKzPDNbbmZLzCzbSzvKzKaa2Xrv/5FeupnZC17Zl5nZWUHbGeItv97MhqSqPKGY2Sgz22lmK4LS4lZGMzvb+wxzvHVTfueVMGV+xMy2eN/1EjMbHPTafV7+15rZFUHpIfd3b9bdeV76P70ZeFPGzI4zsywzW2VmK83sLi89Y7/nVsqcHt9zNLcN60h/BGYZzQVOAg4AlgL9Up2vdpYpD+jSLO0vwDDv8TDgKe/xYOBzwICBwDwv/Shgg/f/SO/xkakuW1B5fgicBaxIRBmB+d6y5q17VZqW+RHgf0Is28/blw8ETvT28U6t7e/AGOBG7/GrwH+muLw9gLO8x4cC67xyZez33EqZ0+J7zsQawDlAjnNug3OuGvgAuCbFeUqEa4C3vcdvA9cGpY92AXOBI8ysB3AFMNU5t9s5VwxMBa5MdqbDcc59BexulhyXMnqvHeacm+sCv5LRQdtKmTBlDuca4APnXJVzbiOQQ2BfD7m/e2e+lwIfeesHf34p4Zzb5pxb5D0uA1YDPcng77mVMoeT1O85EwNATyA/6HkBrX/gHYEDppjZQgvcMhOgu3Num/d4O9Ddexyu/B3xc4lXGXt6j5unp6s7vSaPUQ3NIcRe5qOBPc652mbpacHMegNnAvPwyffcrMyQBt9zJgaATHShc+4s4CrgDjP7YfCL3tlORg/n8kMZPa8AJwP9gW3AiNRmJ/7M7BDgY+D3zrnS4Ncy9XsOUea0+J4zMQBEvO9wR+Oc2+L93wl8QqA6uMOr8uL93+ktHq78HfFziVcZt3iPm6enHefcDudcnXOuHnidwHcNsZd5F4Emk87N0lPKzPYncCB81zk31kvO6O85VJnT5XvOxACQUfcdNrODzezQhsfA5cAKAmVqGP0wBPjUezwOuMUbQTEQKPGq15OBy83sSK+6ebmXls7iUkbvtVIzG+i1md4StK200nAg9FxH4LuGQJlvNLMDzexEoA+BDs+Q+7t3Jp0FXO+tH/z5pYT32b8JrHbOPRv0UsZ+z+HKnDbfcyp7yBP1R2D0wDoCveYPpDo/7SzLSQR6/JcCKxvKQ6DtbxqwHvgCOMpLN2CkV/blwICgbf2aQKdSDnBrqsvWrJzvE6gK1xBox/xNPMsIDPB+ZLnAS3gXQaZhmd/xyrTMOxj0CFr+AS//awka3RJuf/f2nfneZ/EhcGCKy3shgeadZcAS729wJn/PrZQ5Lb5nXQksIuJTmdgEJCIiUVAAEBHxKQUAERGfUgAQEfEpBQAREZ9SABAR8SkFABERn1IAEBHxqf8Pe8I8aU0mjL0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd2bfbbd7f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = [len(x) for x in X_train]\n",
    "plt.plot(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = 200 # specifying the max length of the sequence in the sentence\n",
    "x_filter = []\n",
    "y_filter = []\n",
    "\n",
    "# if the selected length is lesser than the specified max_length, 200, then appending padding (0), else only selecting\n",
    "#       desired length only from sentence\n",
    "for i in range(len(X_train)):\n",
    "    if len(X_train[i])<max_length:\n",
    "        a = len(X_train[i])\n",
    "        X_train[i] = X_train[i] + [0] * (max_length - a)\n",
    "        x_filter.append(X_train[i])\n",
    "        y_filter.append(y_train[i])\n",
    "    elif len(X_train[i])>max_length:\n",
    "        X_train[i] = X_train[i][0:max_length]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#declaring the hyper params\n",
    "embedding_size = 100   # word vector size for initializing the word embeddings\n",
    "n_hidden = 200\n",
    "learning_rate = 0.06\n",
    "training_iters = 100000\n",
    "batch_size = 32\n",
    "beta =0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = max_length #timestepswords\n",
    "n_classes = 2 # 0/1 : binary classification for negative and positive reviews\n",
    "da = 350    #hyper-parameter : Self-attention MLP has hidden layer with da units\n",
    "r = 30      # count of different parts to be extracted from sentence (= # rows in matrix embedding)\n",
    "display_step =10 \n",
    "hidden_units = 3000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.asarray(pd.get_dummies(y_filter))\n",
    "X_train = np.asarray([np.asarray(g) for g in x_filter])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "logs_path = './recent_logs/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataIterator:\n",
    "    def __init__(self, data1,data2, batch_size):\n",
    "        self.data1 = data1\n",
    "        self.data2 = data2\n",
    "        self.batch_size = batch_size\n",
    "        self.iter = self.make_random_iter()\n",
    "        \n",
    "    def next_batch(self):\n",
    "        try:\n",
    "            idxs = next(self.iter)\n",
    "        except StopIteration:\n",
    "            self.iter = self.make_random_iter()\n",
    "            idxs = next(self.iter)\n",
    "        X =[self.data1[i] for i in idxs]\n",
    "        Y =[self.data2[i] for i in idxs]\n",
    "        \n",
    "        X = np.array(X)\n",
    "        Y = np.array(Y)\n",
    "        return X, Y\n",
    "\n",
    "    def make_random_iter(self):\n",
    "        splits = np.arange(self.batch_size, len(self.data1), self.batch_size)\n",
    "        it = np.split(np.random.permutation(range(len(self.data1))), splits)[:-1]\n",
    "        return iter(it)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "############ Graph Creation ################      \n",
    "\n",
    "# TF Graph Input\n",
    "with tf.name_scope(\"weights\"):\n",
    "     Win  = tf.Variable(tf.random_uniform([n_hidden*r, hidden_units],-1/np.sqrt(n_hidden),1/np.sqrt(n_hidden)), name='W-input')\n",
    "     Wout = tf.Variable(tf.random_uniform([hidden_units, n_classes],-1/np.sqrt(hidden_units),1/np.sqrt(hidden_units)), name='W-out')\n",
    "     Ws1  = tf.Variable(tf.random_uniform([da,n_hidden],-1/np.sqrt(da),1/np.sqrt(da)), name='Ws1')\n",
    "     Ws2  = tf.Variable(tf.random_uniform([r,da],-1/np.sqrt(r),1/np.sqrt(r)), name='Ws2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"biases\"):            \n",
    "    biasesout = tf.Variable(tf.random_normal([n_classes]), name='biases-out')\n",
    "    biasesin  = tf.Variable(tf.random_normal([hidden_units]), name='biases-in')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('input'):\n",
    "    x = tf.placeholder(\"int32\", [32,max_length], name='x-input')\n",
    "    y = tf.placeholder(\"int32\", [32, 2], name='y-input')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('embedding'):\n",
    "    embeddings = tf.Variable(tf.random_uniform([vocabulary, embedding_size],-1, 1), name='embeddings')\n",
    "    embed = tf.nn.embedding_lookup(embeddings,x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def length(sequence):\n",
    "    # Computing maximum of elements across dimensions of a tensor\n",
    "    used = tf.sign(tf.reduce_max(tf.abs(sequence), reduction_indices=2))   \n",
    "    \n",
    "    length = tf.reduce_sum(used, reduction_indices=1)\n",
    "    length = tf.cast(length, tf.int32)\n",
    "    return length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.variable_scope('forward',reuse=True):\n",
    "        lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('model'):  \n",
    "    outputs, states = rnn.dynamic_rnn(lstm_fw_cell,embed,sequence_length=length(embed),dtype=tf.float32,time_major=False)    \n",
    "    # in the next step we multiply the hidden-vec matrix with the Ws1 by reshaping \n",
    "    h = tf.nn.tanh(tf.transpose(tf.reshape(tf.matmul(Ws1,tf.reshape(outputs,[n_hidden,batch_size*n_steps])), \n",
    "                                           [da,batch_size,n_steps]),[1,0,2]))\n",
    "    # in this step we multiply the generated matrix with Ws2\n",
    "    a = tf.reshape(tf.matmul(Ws2,tf.reshape(h,[da,batch_size*n_steps])),[batch_size,r,n_steps])\n",
    "    def fn3(a,x):\n",
    "            return tf.nn.softmax(x)\n",
    "    h3 = tf.scan(fn3,a) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('flattening'):\n",
    "    # here we again multiply(batch) of the generated batch with the same hidden matrix\n",
    "    h4 = tf.matmul(h3,outputs)\n",
    "    # flattening the output embedded matrix\n",
    "    last = tf.reshape(h4,[-1,r*n_hidden])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('MLP'):\n",
    "    tf.nn.dropout(last,.5, noise_shape=None, seed=None, name=None)\n",
    "    pred1 = tf.nn.sigmoid(tf.matmul(last,Win)+biasesin)\n",
    "    pred  = tf.matmul(pred1, Wout) + biasesout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "with tf.name_scope('cross'):\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits =pred, labels = y) + beta*tf.nn.l2_loss(Ws2) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('train'):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    gvs = optimizer.compute_gradients(cost)\n",
    "    capped_gvs = [(tf.clip_by_norm(grad,0.5), var) for grad, var in gvs]\n",
    "    optimizer.apply_gradients(capped_gvs)\n",
    "    optimized = optimizer.minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate model\n",
    "with tf.name_scope('Accuracy'):\n",
    "    correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "    accuracy     = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'accuracy:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.summary.scalar(\"cost\", cost)\n",
    "tf.summary.scalar(\"accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all summaries into a single \"summary operation\" which we can execute in a session \n",
    "summary_op =tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializing the variables\n",
    "train_iter = DataIterator(X_train,y_train, batch_size)    \n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# This could give warning if in case the required port is being used already\n",
    "# Running the command again or releasing the port before the subsequent run should solve the purpose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 64, Minibatch Loss= 68.048653, Training Accuracy= 50.00%\n",
      "Iter 384, Minibatch Loss= 69.634018, Training Accuracy= 53.12%\n",
      "Iter 704, Minibatch Loss= 50.814949, Training Accuracy= 46.88%\n",
      "Iter 1024, Minibatch Loss= 39.475891, Training Accuracy= 56.25%\n",
      "Iter 1344, Minibatch Loss= 11.115482, Training Accuracy= 40.62%\n",
      "Iter 1664, Minibatch Loss= 7.060193, Training Accuracy= 59.38%\n",
      "Iter 1984, Minibatch Loss= 2.565218, Training Accuracy= 43.75%\n",
      "Iter 2304, Minibatch Loss= 18.036911, Training Accuracy= 46.88%\n",
      "Iter 2624, Minibatch Loss= 18.796995, Training Accuracy= 43.75%\n",
      "Iter 2944, Minibatch Loss= 56.627518, Training Accuracy= 43.75%\n",
      "Iter 3264, Minibatch Loss= 29.162407, Training Accuracy= 43.75%\n",
      "Iter 3584, Minibatch Loss= 14.335728, Training Accuracy= 40.62%\n",
      "Iter 3904, Minibatch Loss= 1.863467, Training Accuracy= 53.12%\n",
      "Iter 4224, Minibatch Loss= 7.892468, Training Accuracy= 50.00%\n",
      "Iter 4544, Minibatch Loss= 4.554517, Training Accuracy= 53.12%\n",
      "Iter 4864, Minibatch Loss= 19.358366, Training Accuracy= 56.25%\n",
      "Iter 5184, Minibatch Loss= 8.660361, Training Accuracy= 25.00%\n",
      "Iter 5504, Minibatch Loss= 8.641634, Training Accuracy= 43.75%\n",
      "Iter 5824, Minibatch Loss= 4.625659, Training Accuracy= 68.75%\n",
      "Iter 6144, Minibatch Loss= 10.283667, Training Accuracy= 53.12%\n",
      "Iter 6464, Minibatch Loss= 8.946012, Training Accuracy= 65.62%\n",
      "Iter 6784, Minibatch Loss= 10.141964, Training Accuracy= 34.38%\n",
      "Iter 7104, Minibatch Loss= 28.192059, Training Accuracy= 46.88%\n",
      "Iter 7424, Minibatch Loss= 4.486736, Training Accuracy= 56.25%\n",
      "Iter 7744, Minibatch Loss= 21.987207, Training Accuracy= 56.25%\n",
      "Iter 8064, Minibatch Loss= 4.697628, Training Accuracy= 62.50%\n",
      "Iter 8384, Minibatch Loss= 4.270442, Training Accuracy= 59.38%\n",
      "Iter 8704, Minibatch Loss= 28.705460, Training Accuracy= 40.62%\n",
      "Iter 9024, Minibatch Loss= 1.068065, Training Accuracy= 62.50%\n",
      "Iter 9344, Minibatch Loss= 17.426367, Training Accuracy= 40.62%\n",
      "Iter 9664, Minibatch Loss= 3.467830, Training Accuracy= 43.75%\n",
      "Iter 9984, Minibatch Loss= 15.951517, Training Accuracy= 53.12%\n",
      "Iter 10304, Minibatch Loss= 27.768030, Training Accuracy= 56.25%\n",
      "Iter 10624, Minibatch Loss= 23.670208, Training Accuracy= 56.25%\n",
      "Iter 10944, Minibatch Loss= 16.554707, Training Accuracy= 34.38%\n",
      "Iter 11264, Minibatch Loss= 1.862891, Training Accuracy= 53.12%\n",
      "Iter 11584, Minibatch Loss= 14.394958, Training Accuracy= 46.88%\n",
      "Iter 11904, Minibatch Loss= 14.636830, Training Accuracy= 59.38%\n",
      "Iter 12224, Minibatch Loss= 22.526840, Training Accuracy= 50.00%\n",
      "Iter 12544, Minibatch Loss= 11.277531, Training Accuracy= 46.88%\n",
      "Iter 12864, Minibatch Loss= 8.312257, Training Accuracy= 59.38%\n",
      "Iter 13184, Minibatch Loss= 2.571780, Training Accuracy= 43.75%\n",
      "Iter 13504, Minibatch Loss= 7.972239, Training Accuracy= 62.50%\n",
      "Iter 13824, Minibatch Loss= 4.762474, Training Accuracy= 50.00%\n",
      "Iter 14144, Minibatch Loss= 8.425671, Training Accuracy= 62.50%\n",
      "Iter 14464, Minibatch Loss= 11.759327, Training Accuracy= 53.12%\n",
      "Iter 14784, Minibatch Loss= 7.914523, Training Accuracy= 50.00%\n",
      "Iter 15104, Minibatch Loss= 10.796078, Training Accuracy= 53.12%\n",
      "Iter 15424, Minibatch Loss= 7.343972, Training Accuracy= 46.88%\n",
      "Iter 15744, Minibatch Loss= 2.848967, Training Accuracy= 62.50%\n",
      "Iter 16064, Minibatch Loss= 13.384836, Training Accuracy= 53.12%\n",
      "Iter 16384, Minibatch Loss= 0.764324, Training Accuracy= 62.50%\n",
      "Iter 16704, Minibatch Loss= 3.935778, Training Accuracy= 59.38%\n",
      "Iter 17024, Minibatch Loss= 7.596835, Training Accuracy= 50.00%\n",
      "Iter 17344, Minibatch Loss= 6.328508, Training Accuracy= 68.75%\n",
      "Iter 17664, Minibatch Loss= 17.243980, Training Accuracy= 50.00%\n",
      "Iter 17984, Minibatch Loss= 3.188797, Training Accuracy= 62.50%\n",
      "Iter 18304, Minibatch Loss= 4.553954, Training Accuracy= 53.12%\n",
      "Iter 18624, Minibatch Loss= 6.739563, Training Accuracy= 53.12%\n",
      "Iter 18944, Minibatch Loss= 13.548286, Training Accuracy= 53.12%\n",
      "Iter 19264, Minibatch Loss= 9.630369, Training Accuracy= 59.38%\n",
      "Iter 19584, Minibatch Loss= 14.731222, Training Accuracy= 50.00%\n",
      "Iter 19904, Minibatch Loss= 13.236647, Training Accuracy= 31.25%\n",
      "Iter 20224, Minibatch Loss= 12.380626, Training Accuracy= 53.12%\n",
      "Iter 20544, Minibatch Loss= 6.639822, Training Accuracy= 56.25%\n",
      "Iter 20864, Minibatch Loss= 13.415706, Training Accuracy= 37.50%\n",
      "Iter 21184, Minibatch Loss= 4.680547, Training Accuracy= 62.50%\n",
      "Iter 21504, Minibatch Loss= 14.221049, Training Accuracy= 56.25%\n",
      "Iter 21824, Minibatch Loss= 16.064651, Training Accuracy= 43.75%\n",
      "Iter 22144, Minibatch Loss= 4.457575, Training Accuracy= 40.62%\n",
      "Iter 22464, Minibatch Loss= 2.053376, Training Accuracy= 59.38%\n",
      "Iter 22784, Minibatch Loss= 20.164852, Training Accuracy= 34.38%\n",
      "Iter 23104, Minibatch Loss= 8.480515, Training Accuracy= 53.12%\n",
      "Iter 23424, Minibatch Loss= 7.034980, Training Accuracy= 53.12%\n",
      "Iter 23744, Minibatch Loss= 4.895866, Training Accuracy= 43.75%\n",
      "Iter 24064, Minibatch Loss= 15.955807, Training Accuracy= 43.75%\n",
      "Iter 24384, Minibatch Loss= 3.094639, Training Accuracy= 46.88%\n",
      "Iter 24704, Minibatch Loss= 1.274401, Training Accuracy= 43.75%\n",
      "Iter 25024, Minibatch Loss= 0.661598, Training Accuracy= 59.38%\n",
      "Iter 25344, Minibatch Loss= 37.563221, Training Accuracy= 43.75%\n",
      "Iter 25664, Minibatch Loss= 2.874022, Training Accuracy= 50.00%\n",
      "Iter 25984, Minibatch Loss= 11.354934, Training Accuracy= 46.88%\n",
      "Iter 26304, Minibatch Loss= 8.918873, Training Accuracy= 34.38%\n",
      "Iter 26624, Minibatch Loss= 6.128410, Training Accuracy= 59.38%\n",
      "Iter 26944, Minibatch Loss= 5.735064, Training Accuracy= 53.12%\n",
      "Iter 27264, Minibatch Loss= 9.497025, Training Accuracy= 43.75%\n",
      "Iter 27584, Minibatch Loss= 4.762102, Training Accuracy= 50.00%\n",
      "Iter 27904, Minibatch Loss= 2.539943, Training Accuracy= 50.00%\n",
      "Iter 28224, Minibatch Loss= 22.644489, Training Accuracy= 53.12%\n",
      "Iter 28544, Minibatch Loss= 3.768353, Training Accuracy= 37.50%\n",
      "Iter 28864, Minibatch Loss= 1.592333, Training Accuracy= 56.25%\n",
      "Iter 29184, Minibatch Loss= 0.833674, Training Accuracy= 50.00%\n",
      "Iter 29504, Minibatch Loss= 8.278477, Training Accuracy= 56.25%\n",
      "Iter 29824, Minibatch Loss= 8.094858, Training Accuracy= 40.62%\n",
      "Iter 30144, Minibatch Loss= 13.986859, Training Accuracy= 53.12%\n",
      "Iter 30464, Minibatch Loss= 14.349034, Training Accuracy= 40.62%\n",
      "Iter 30784, Minibatch Loss= 3.889781, Training Accuracy= 56.25%\n",
      "Iter 31104, Minibatch Loss= 21.634075, Training Accuracy= 53.12%\n",
      "Iter 31424, Minibatch Loss= 4.705059, Training Accuracy= 56.25%\n",
      "Iter 31744, Minibatch Loss= 9.733704, Training Accuracy= 43.75%\n",
      "Iter 32064, Minibatch Loss= 4.969881, Training Accuracy= 53.12%\n",
      "Iter 32384, Minibatch Loss= 2.800320, Training Accuracy= 46.88%\n",
      "Iter 32704, Minibatch Loss= 5.603562, Training Accuracy= 46.88%\n",
      "Iter 33024, Minibatch Loss= 2.687968, Training Accuracy= 50.00%\n",
      "Iter 33344, Minibatch Loss= 18.012241, Training Accuracy= 40.62%\n",
      "Iter 33664, Minibatch Loss= 10.452724, Training Accuracy= 65.62%\n",
      "Iter 33984, Minibatch Loss= 13.324929, Training Accuracy= 46.88%\n",
      "Iter 34304, Minibatch Loss= 19.693317, Training Accuracy= 46.88%\n",
      "Iter 34624, Minibatch Loss= 4.306432, Training Accuracy= 53.12%\n",
      "Iter 34944, Minibatch Loss= 7.429633, Training Accuracy= 53.12%\n",
      "Iter 35264, Minibatch Loss= 3.740365, Training Accuracy= 37.50%\n",
      "Iter 35584, Minibatch Loss= 15.840183, Training Accuracy= 43.75%\n",
      "Iter 35904, Minibatch Loss= 15.707689, Training Accuracy= 59.38%\n",
      "Iter 36224, Minibatch Loss= 10.785508, Training Accuracy= 40.62%\n",
      "Iter 36544, Minibatch Loss= 7.691505, Training Accuracy= 34.38%\n",
      "Iter 36864, Minibatch Loss= 27.035702, Training Accuracy= 34.38%\n",
      "Iter 37184, Minibatch Loss= 4.179764, Training Accuracy= 56.25%\n",
      "Iter 37504, Minibatch Loss= 0.824007, Training Accuracy= 40.62%\n",
      "Iter 37824, Minibatch Loss= 6.629670, Training Accuracy= 56.25%\n",
      "Iter 38144, Minibatch Loss= 12.860003, Training Accuracy= 43.75%\n",
      "Iter 38464, Minibatch Loss= 6.448372, Training Accuracy= 40.62%\n",
      "Iter 38784, Minibatch Loss= 9.507837, Training Accuracy= 59.38%\n",
      "Iter 39104, Minibatch Loss= 12.370415, Training Accuracy= 59.38%\n",
      "Iter 39424, Minibatch Loss= 16.644466, Training Accuracy= 40.62%\n",
      "Iter 39744, Minibatch Loss= 10.079248, Training Accuracy= 46.88%\n",
      "Iter 40064, Minibatch Loss= 1.594822, Training Accuracy= 46.88%\n",
      "Iter 40384, Minibatch Loss= 9.232342, Training Accuracy= 43.75%\n",
      "Iter 40704, Minibatch Loss= 5.345547, Training Accuracy= 50.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 41024, Minibatch Loss= 21.031528, Training Accuracy= 43.75%\n",
      "Iter 41344, Minibatch Loss= 23.064739, Training Accuracy= 46.88%\n",
      "Iter 41664, Minibatch Loss= 15.065256, Training Accuracy= 40.62%\n",
      "Iter 41984, Minibatch Loss= 17.597385, Training Accuracy= 56.25%\n",
      "Iter 42304, Minibatch Loss= 2.784813, Training Accuracy= 53.12%\n",
      "Iter 42624, Minibatch Loss= 1.562894, Training Accuracy= 40.62%\n",
      "Iter 42944, Minibatch Loss= 12.699908, Training Accuracy= 46.88%\n",
      "Iter 43264, Minibatch Loss= 4.876400, Training Accuracy= 40.62%\n",
      "Iter 43584, Minibatch Loss= 4.536665, Training Accuracy= 53.12%\n",
      "Iter 43904, Minibatch Loss= 2.228184, Training Accuracy= 59.38%\n",
      "Iter 44224, Minibatch Loss= 5.214026, Training Accuracy= 59.38%\n",
      "Iter 44544, Minibatch Loss= 4.667718, Training Accuracy= 40.62%\n",
      "Iter 44864, Minibatch Loss= 2.383085, Training Accuracy= 53.12%\n",
      "Iter 45184, Minibatch Loss= 6.075849, Training Accuracy= 50.00%\n",
      "Iter 45504, Minibatch Loss= 1.285813, Training Accuracy= 56.25%\n",
      "Iter 45824, Minibatch Loss= 4.313162, Training Accuracy= 43.75%\n",
      "Iter 46144, Minibatch Loss= 4.176050, Training Accuracy= 59.38%\n",
      "Iter 46464, Minibatch Loss= 16.904991, Training Accuracy= 62.50%\n",
      "Iter 46784, Minibatch Loss= 1.465783, Training Accuracy= 62.50%\n",
      "Iter 47104, Minibatch Loss= 15.791449, Training Accuracy= 50.00%\n",
      "Iter 47424, Minibatch Loss= 20.306522, Training Accuracy= 59.38%\n",
      "Iter 47744, Minibatch Loss= 25.076126, Training Accuracy= 50.00%\n",
      "Iter 48064, Minibatch Loss= 13.938768, Training Accuracy= 65.62%\n",
      "Iter 48384, Minibatch Loss= 7.911925, Training Accuracy= 50.00%\n",
      "Iter 48704, Minibatch Loss= 4.146496, Training Accuracy= 56.25%\n",
      "Iter 49024, Minibatch Loss= 8.021297, Training Accuracy= 53.12%\n",
      "Iter 49344, Minibatch Loss= 2.008795, Training Accuracy= 53.12%\n",
      "Iter 49664, Minibatch Loss= 3.912958, Training Accuracy= 53.12%\n",
      "Iter 49984, Minibatch Loss= 11.316398, Training Accuracy= 53.12%\n",
      "Iter 50304, Minibatch Loss= 2.039451, Training Accuracy= 46.88%\n",
      "Iter 50624, Minibatch Loss= 16.229889, Training Accuracy= 46.88%\n",
      "Iter 50944, Minibatch Loss= 8.432322, Training Accuracy= 43.75%\n",
      "Iter 51264, Minibatch Loss= 2.534712, Training Accuracy= 56.25%\n",
      "Iter 51584, Minibatch Loss= 31.529041, Training Accuracy= 40.62%\n",
      "Iter 51904, Minibatch Loss= 11.681208, Training Accuracy= 43.75%\n",
      "Iter 52224, Minibatch Loss= 23.440483, Training Accuracy= 40.62%\n",
      "Iter 52544, Minibatch Loss= 0.666839, Training Accuracy= 62.50%\n",
      "Iter 52864, Minibatch Loss= 21.388758, Training Accuracy= 40.62%\n",
      "Iter 53184, Minibatch Loss= 12.126427, Training Accuracy= 46.88%\n",
      "Iter 53504, Minibatch Loss= 12.451681, Training Accuracy= 50.00%\n",
      "Iter 53824, Minibatch Loss= 3.985178, Training Accuracy= 43.75%\n",
      "Iter 54144, Minibatch Loss= 9.805837, Training Accuracy= 40.62%\n",
      "Iter 54464, Minibatch Loss= 14.656006, Training Accuracy= 46.88%\n",
      "Iter 54784, Minibatch Loss= 5.890873, Training Accuracy= 43.75%\n",
      "Iter 55104, Minibatch Loss= 0.772709, Training Accuracy= 46.88%\n",
      "Iter 55424, Minibatch Loss= 3.218151, Training Accuracy= 50.00%\n",
      "Iter 55744, Minibatch Loss= 17.635784, Training Accuracy= 40.62%\n",
      "Iter 56064, Minibatch Loss= 8.105782, Training Accuracy= 62.50%\n",
      "Iter 56384, Minibatch Loss= 6.164281, Training Accuracy= 59.38%\n",
      "Iter 56704, Minibatch Loss= 3.506629, Training Accuracy= 59.38%\n",
      "Iter 57024, Minibatch Loss= 6.995962, Training Accuracy= 68.75%\n",
      "Iter 57344, Minibatch Loss= 8.790695, Training Accuracy= 43.75%\n",
      "Iter 57664, Minibatch Loss= 4.156086, Training Accuracy= 53.12%\n",
      "Iter 57984, Minibatch Loss= 9.201471, Training Accuracy= 62.50%\n",
      "Iter 58304, Minibatch Loss= 10.226985, Training Accuracy= 53.12%\n",
      "Iter 58624, Minibatch Loss= 2.116041, Training Accuracy= 37.50%\n",
      "Iter 58944, Minibatch Loss= 6.928905, Training Accuracy= 34.38%\n",
      "Iter 59264, Minibatch Loss= 9.766208, Training Accuracy= 46.88%\n",
      "Iter 59584, Minibatch Loss= 20.821720, Training Accuracy= 65.62%\n",
      "Iter 59904, Minibatch Loss= 24.298084, Training Accuracy= 53.12%\n",
      "Iter 60224, Minibatch Loss= 7.316458, Training Accuracy= 43.75%\n",
      "Iter 60544, Minibatch Loss= 6.037505, Training Accuracy= 53.12%\n",
      "Iter 60864, Minibatch Loss= 0.721190, Training Accuracy= 50.00%\n",
      "Iter 61184, Minibatch Loss= 2.178830, Training Accuracy= 50.00%\n",
      "Iter 61504, Minibatch Loss= 7.628143, Training Accuracy= 65.62%\n",
      "Iter 61824, Minibatch Loss= 3.177518, Training Accuracy= 59.38%\n",
      "Iter 62144, Minibatch Loss= 10.251677, Training Accuracy= 53.12%\n",
      "Iter 62464, Minibatch Loss= 8.555613, Training Accuracy= 59.38%\n",
      "Iter 62784, Minibatch Loss= 5.271868, Training Accuracy= 40.62%\n",
      "Iter 63104, Minibatch Loss= 19.958477, Training Accuracy= 43.75%\n",
      "Iter 63424, Minibatch Loss= 15.127458, Training Accuracy= 50.00%\n",
      "Iter 63744, Minibatch Loss= 1.812528, Training Accuracy= 50.00%\n",
      "Iter 64064, Minibatch Loss= 12.482746, Training Accuracy= 53.12%\n",
      "Iter 64384, Minibatch Loss= 26.122999, Training Accuracy= 50.00%\n",
      "Iter 64704, Minibatch Loss= 30.161316, Training Accuracy= 43.75%\n",
      "Iter 65024, Minibatch Loss= 4.300615, Training Accuracy= 65.62%\n",
      "Iter 65344, Minibatch Loss= 0.784370, Training Accuracy= 71.88%\n",
      "Iter 65664, Minibatch Loss= 9.419561, Training Accuracy= 62.50%\n",
      "Iter 65984, Minibatch Loss= 23.746841, Training Accuracy= 43.75%\n",
      "Iter 66304, Minibatch Loss= 21.083220, Training Accuracy= 43.75%\n",
      "Iter 66624, Minibatch Loss= 16.359665, Training Accuracy= 62.50%\n",
      "Iter 66944, Minibatch Loss= 8.083037, Training Accuracy= 53.12%\n",
      "Iter 67264, Minibatch Loss= 5.965985, Training Accuracy= 53.12%\n",
      "Iter 67584, Minibatch Loss= 2.470879, Training Accuracy= 46.88%\n",
      "Iter 67904, Minibatch Loss= 9.042778, Training Accuracy= 56.25%\n",
      "Iter 68224, Minibatch Loss= 1.788287, Training Accuracy= 46.88%\n",
      "Iter 68544, Minibatch Loss= 14.053770, Training Accuracy= 50.00%\n",
      "Iter 68864, Minibatch Loss= 12.774237, Training Accuracy= 62.50%\n",
      "Iter 69184, Minibatch Loss= 2.655466, Training Accuracy= 50.00%\n",
      "Iter 69504, Minibatch Loss= 1.799385, Training Accuracy= 46.88%\n",
      "Iter 69824, Minibatch Loss= 16.016491, Training Accuracy= 43.75%\n",
      "Iter 70144, Minibatch Loss= 10.564743, Training Accuracy= 59.38%\n",
      "Iter 70464, Minibatch Loss= 15.462074, Training Accuracy= 56.25%\n",
      "Iter 70784, Minibatch Loss= 17.902014, Training Accuracy= 34.38%\n",
      "Iter 71104, Minibatch Loss= 15.256748, Training Accuracy= 40.62%\n",
      "Iter 71424, Minibatch Loss= 1.235040, Training Accuracy= 50.00%\n",
      "Iter 71744, Minibatch Loss= 1.084303, Training Accuracy= 56.25%\n",
      "Iter 72064, Minibatch Loss= 2.071187, Training Accuracy= 46.88%\n",
      "Iter 72384, Minibatch Loss= 14.241503, Training Accuracy= 43.75%\n",
      "Iter 72704, Minibatch Loss= 0.710391, Training Accuracy= 56.25%\n",
      "Iter 73024, Minibatch Loss= 6.935651, Training Accuracy= 40.62%\n",
      "Iter 73344, Minibatch Loss= 1.152113, Training Accuracy= 40.62%\n",
      "Iter 73664, Minibatch Loss= 3.757454, Training Accuracy= 40.62%\n",
      "Iter 73984, Minibatch Loss= 3.017574, Training Accuracy= 59.38%\n",
      "Iter 74304, Minibatch Loss= 2.525484, Training Accuracy= 50.00%\n",
      "Iter 74624, Minibatch Loss= 14.177895, Training Accuracy= 56.25%\n",
      "Iter 74944, Minibatch Loss= 23.525188, Training Accuracy= 56.25%\n",
      "Iter 75264, Minibatch Loss= 11.270742, Training Accuracy= 56.25%\n",
      "Iter 75584, Minibatch Loss= 23.212002, Training Accuracy= 46.88%\n",
      "Iter 75904, Minibatch Loss= 19.986408, Training Accuracy= 46.88%\n",
      "Iter 76224, Minibatch Loss= 5.351293, Training Accuracy= 56.25%\n",
      "Iter 76544, Minibatch Loss= 3.647470, Training Accuracy= 59.38%\n",
      "Iter 76864, Minibatch Loss= 4.175218, Training Accuracy= 53.12%\n",
      "Iter 77184, Minibatch Loss= 4.536756, Training Accuracy= 59.38%\n",
      "Iter 77504, Minibatch Loss= 2.010255, Training Accuracy= 56.25%\n",
      "Iter 77824, Minibatch Loss= 2.634599, Training Accuracy= 62.50%\n",
      "Iter 78144, Minibatch Loss= 8.022557, Training Accuracy= 53.12%\n",
      "Iter 78464, Minibatch Loss= 8.778473, Training Accuracy= 34.38%\n",
      "Iter 78784, Minibatch Loss= 3.060512, Training Accuracy= 59.38%\n",
      "Iter 79104, Minibatch Loss= 1.343833, Training Accuracy= 53.12%\n",
      "Iter 79424, Minibatch Loss= 14.035875, Training Accuracy= 53.12%\n",
      "Iter 79744, Minibatch Loss= 1.705568, Training Accuracy= 46.88%\n",
      "Iter 80064, Minibatch Loss= 6.593272, Training Accuracy= 53.12%\n",
      "Iter 80384, Minibatch Loss= 15.696155, Training Accuracy= 59.38%\n",
      "Iter 80704, Minibatch Loss= 22.825464, Training Accuracy= 59.38%\n",
      "Iter 81024, Minibatch Loss= 20.275944, Training Accuracy= 46.88%\n",
      "Iter 81344, Minibatch Loss= 12.754831, Training Accuracy= 56.25%\n",
      "Iter 81664, Minibatch Loss= 7.261662, Training Accuracy= 40.62%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 81984, Minibatch Loss= 7.793962, Training Accuracy= 50.00%\n",
      "Iter 82304, Minibatch Loss= 0.721587, Training Accuracy= 50.00%\n",
      "Iter 82624, Minibatch Loss= 16.602007, Training Accuracy= 50.00%\n",
      "Iter 82944, Minibatch Loss= 33.046600, Training Accuracy= 46.88%\n",
      "Iter 83264, Minibatch Loss= 11.703110, Training Accuracy= 65.62%\n",
      "Iter 83584, Minibatch Loss= 9.445695, Training Accuracy= 34.38%\n",
      "Iter 83904, Minibatch Loss= 15.244149, Training Accuracy= 59.38%\n",
      "Iter 84224, Minibatch Loss= 3.159154, Training Accuracy= 50.00%\n",
      "Iter 84544, Minibatch Loss= 28.657928, Training Accuracy= 50.00%\n",
      "Iter 84864, Minibatch Loss= 8.399003, Training Accuracy= 53.12%\n",
      "Iter 85184, Minibatch Loss= 14.731194, Training Accuracy= 46.88%\n",
      "Iter 85504, Minibatch Loss= 5.818152, Training Accuracy= 62.50%\n",
      "Iter 85824, Minibatch Loss= 19.275805, Training Accuracy= 50.00%\n",
      "Iter 86144, Minibatch Loss= 10.736831, Training Accuracy= 53.12%\n",
      "Iter 86464, Minibatch Loss= 9.993341, Training Accuracy= 50.00%\n",
      "Iter 86784, Minibatch Loss= 4.928855, Training Accuracy= 43.75%\n",
      "Iter 87104, Minibatch Loss= 1.523685, Training Accuracy= 53.12%\n",
      "Iter 87424, Minibatch Loss= 7.992968, Training Accuracy= 40.62%\n",
      "Iter 87744, Minibatch Loss= 13.455222, Training Accuracy= 37.50%\n",
      "Iter 88064, Minibatch Loss= 6.928088, Training Accuracy= 62.50%\n",
      "Iter 88384, Minibatch Loss= 5.722057, Training Accuracy= 59.38%\n",
      "Iter 88704, Minibatch Loss= 19.526503, Training Accuracy= 46.88%\n",
      "Iter 89024, Minibatch Loss= 12.010965, Training Accuracy= 31.25%\n",
      "Iter 89344, Minibatch Loss= 12.129732, Training Accuracy= 40.62%\n",
      "Iter 89664, Minibatch Loss= 1.129440, Training Accuracy= 40.62%\n",
      "Iter 89984, Minibatch Loss= 1.790181, Training Accuracy= 37.50%\n",
      "Iter 90304, Minibatch Loss= 2.876821, Training Accuracy= 40.62%\n",
      "Iter 90624, Minibatch Loss= 18.299339, Training Accuracy= 56.25%\n",
      "Iter 90944, Minibatch Loss= 7.193353, Training Accuracy= 53.12%\n",
      "Iter 91264, Minibatch Loss= 15.776569, Training Accuracy= 37.50%\n",
      "Iter 91584, Minibatch Loss= 14.444285, Training Accuracy= 43.75%\n",
      "Iter 91904, Minibatch Loss= 43.865486, Training Accuracy= 43.75%\n",
      "Iter 92224, Minibatch Loss= 15.423979, Training Accuracy= 50.00%\n",
      "Iter 92544, Minibatch Loss= 11.372643, Training Accuracy= 25.00%\n",
      "Iter 92864, Minibatch Loss= 1.179256, Training Accuracy= 50.00%\n",
      "Iter 93184, Minibatch Loss= 14.395565, Training Accuracy= 43.75%\n",
      "Iter 93504, Minibatch Loss= 1.220622, Training Accuracy= 56.25%\n",
      "Iter 93824, Minibatch Loss= 5.193919, Training Accuracy= 56.25%\n",
      "Iter 94144, Minibatch Loss= 5.515789, Training Accuracy= 53.12%\n",
      "Iter 94464, Minibatch Loss= 0.745686, Training Accuracy= 50.00%\n",
      "Iter 94784, Minibatch Loss= 4.399930, Training Accuracy= 40.62%\n",
      "Iter 95104, Minibatch Loss= 12.189066, Training Accuracy= 59.38%\n",
      "Iter 95424, Minibatch Loss= 13.075596, Training Accuracy= 56.25%\n",
      "Iter 95744, Minibatch Loss= 28.283163, Training Accuracy= 59.38%\n",
      "Iter 96064, Minibatch Loss= 1.305542, Training Accuracy= 50.00%\n",
      "Iter 96384, Minibatch Loss= 1.801988, Training Accuracy= 50.00%\n",
      "Iter 96704, Minibatch Loss= 1.896597, Training Accuracy= 53.12%\n",
      "Iter 97024, Minibatch Loss= 2.941552, Training Accuracy= 46.88%\n",
      "Iter 97344, Minibatch Loss= 0.693964, Training Accuracy= 56.25%\n",
      "Iter 97664, Minibatch Loss= 8.340314, Training Accuracy= 40.62%\n",
      "Iter 97984, Minibatch Loss= 2.635653, Training Accuracy= 56.25%\n",
      "Iter 98304, Minibatch Loss= 1.541869, Training Accuracy= 68.75%\n",
      "Iter 98624, Minibatch Loss= 1.544908, Training Accuracy= 62.50%\n",
      "Iter 98944, Minibatch Loss= 26.138868, Training Accuracy= 56.25%\n",
      "Iter 99264, Minibatch Loss= 17.603979, Training Accuracy= 56.25%\n",
      "Iter 99584, Minibatch Loss= 21.715031, Training Accuracy= 40.62%\n",
      "Iter 99904, Minibatch Loss= 17.485657, Training Accuracy= 53.12%\n",
      "Optimization Finished!\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    # Creating log file writer object\n",
    "    writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())\n",
    "    step = 1\n",
    "    \n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_x, batch_y = train_iter.next_batch()\n",
    "        sess.run(optimized, feed_dict={x: batch_x, y: batch_y})\n",
    "        # Executing the summary operation in the session\n",
    "        summary = sess.run(summary_op, feed_dict={x: batch_x, y: batch_y})\n",
    "        # Writing the values in log file using the FileWriter object created above\n",
    "        writer.add_summary(summary,  step*batch_size)\n",
    "        if step % display_step == 2:\n",
    "            # Calculate batch accuracy\n",
    "            acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n",
    "            # Calculate batch loss\n",
    "            loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n",
    "            print (\"Iter \" + str(step*batch_size) + \\\n",
    "                   \", Minibatch Loss= \" + \"{:.6f}\".format(loss) + \\\n",
    "                   \", Training Accuracy= \" + \"{:.2f}\".format(acc*100) + \"%\")\n",
    "        step += 1\n",
    "    print (\"Optimization Finished!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
